{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       id  score                                               fact  \\\n",
      "0  erv4zf   6928  In Avengers Endgame (2019), Tony Starks suit r...   \n",
      "1  erxtri   1222  In Ice Age (2002) Sid is singing about how all...   \n",
      "2  es1t17    708  In Star Wars: The Force Awakens (2015) Rey's g...   \n",
      "3  eruyie    595  In Shazam (2019) director David Sandberg, who ...   \n",
      "4  es2xzs    559  Boba Fett recognises R2-D2 on Jabba's Sail Bar...   \n",
      "\n",
      "        topic  \n",
      "0  Literature  \n",
      "1       Music  \n",
      "2   Movies_TV  \n",
      "3   Movies_TV  \n",
      "4   Movies_TV  \n",
      "Total non-unique nounphrases: 37045\n",
      "Total unique nounphrases: 21602\n",
      "Number of nounphrases: 13054\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "import os\n",
    "import string\n",
    "import re\n",
    "from nltk.tokenize import sent_tokenize\n",
    "import pandas as pd\n",
    "import argparse\n",
    "from copy import deepcopy\n",
    "import requests\n",
    "\n",
    "# parser = argparse.ArgumentParser()\n",
    "\n",
    "# parser.add_argument(\"mode\", help=\"questions or facts\", type=str)\n",
    "mode = \"facts\"\n",
    "\n",
    "SERVICE_URL = 'http://0.0.0.0:8016/nounphrases'\n",
    "\n",
    "np_ignore_list = [\"'s\", 'i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', \"you're\",\n",
    "                  \"you've\", \"you'll\", \"you'd\", 'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself',\n",
    "                  'she', \"she's\", 'her', 'hers', 'herself', 'it', \"it's\", 'its', 'itself', 'they', 'them', 'their',\n",
    "                  'theirs', 'themselves', 'what', 'which', 'who', 'whom', 'this', 'that', \"that'll\", 'these', 'those',\n",
    "                  'am', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does',\n",
    "                  'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', 'as', 'until', 'while', 'of',\n",
    "                  'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into', 'through', 'during', 'before',\n",
    "                  'after', 'above', 'below', 'to', 'from', 'up', 'down', 'in', 'out', 'on', 'off', 'over', 'under',\n",
    "                  'again', 'further', 'then', 'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'any',\n",
    "                  'both', 'each', 'few', 'more', 'most', 'other', 'some', 'such', 'no', 'nor', 'not', 'only', 'own',\n",
    "                  'same', 'so', 'than', 'too', 'very', 's', 't', 'can', 'will', 'just', 'don', \"don't\", 'should',\n",
    "                  \"should've\", 'now', 'd', 'll', 'm', 'o', 're', 've', 'y', 'ain', 'aren', \"aren't\", 'couldn',\n",
    "                  \"couldn't\", 'didn', \"didn't\", 'doesn', \"doesn't\", 'hadn', \"hadn't\", 'hasn', \"hasn't\", 'haven',\n",
    "                  \"haven't\", 'isn', \"isn't\", 'ma', 'mightn', \"mightn't\", 'mustn', \"mustn't\", 'needn', \"needn't\",\n",
    "                  'shan', \"shan't\", 'shouldn', \"shouldn't\", 'wasn', \"wasn't\", 'weren', \"weren't\", 'won', \"won't\",\n",
    "                  'wouldn', \"wouldn't\", \"my name\", \"your name\", \"wow\", \"yeah\", \"yes\", \"ya\", \"cool\", \"okay\", \"more\",\n",
    "                  \"some more\", \" a lot\", \"a bit\", \"another one\", \"something else\", \"something\", \"anything\",\n",
    "                  \"someone\", \"anyone\", \"play\", \"mean\", \"a lot\", \"a little\", \"a little bit\"]\n",
    "\n",
    "df = pd.read_csv(f\"{mode}_with_topics.csv\")\n",
    "print(df.head())\n",
    "    \n",
    "df_dict = []\n",
    "\n",
    "for i in range(df.shape[0]):\n",
    "    df_dict.append({mode[:-1]: df.loc[i, mode[:-1]], \"topic\": df.loc[i, \"topic\"]})\n",
    "\n",
    "batch_size = 100\n",
    "all_topics = []\n",
    "\n",
    "for i in range(int(len(df_dict) / batch_size) + 1):\n",
    "    dialogs = [deepcopy({\"utterances\": [{\"text\": \"\"}]}) for _ in df_dict[i * batch_size: (i+1)* batch_size]] \n",
    "\n",
    "    for dialog, el in zip(dialogs, df_dict[i * batch_size: (i+1)* batch_size]):\n",
    "        dialog[\"utterances\"][-1][\"text\"] = el[mode[:-1]]\n",
    "    try:\n",
    "        nounphrases = requests.request(url=SERVICE_URL, json={'dialogs': dialogs}, method='POST').json()\n",
    "    except:\n",
    "        print(dialogs)\n",
    "        continue\n",
    "\n",
    "    for el, np in zip(df_dict[i * batch_size: (i+1)* batch_size], nounphrases):\n",
    "        el[\"nounphrases\"] = np\n",
    "        \n",
    "# extract unique noun phrases from dataet\n",
    "unique_nps = []\n",
    "spaces = re.compile(\"\\s\\s+\")\n",
    "ignore_np_res = []\n",
    "for ignore_np in np_ignore_list:\n",
    "    ignore_np_res.append(re.compile(r'\\b%s\\b' % ignore_np))\n",
    "\n",
    "for sample in df_dict:\n",
    "    for np in sample[\"nounphrases\"]:\n",
    "        for ignore_np in ignore_np_res:\n",
    "            np = re.sub(spaces, \" \", re.sub(ignore_np, \"\", np)).strip()\n",
    "        if len(np) >= 3:\n",
    "            unique_nps.append(np)\n",
    "print(f\"Total non-unique nounphrases: {len(unique_nps)}\")\n",
    "unique_nps = list(set(unique_nps))\n",
    "print(f\"Total unique nounphrases: {len(unique_nps)}\")\n",
    "\n",
    "np_to_fact_map = {}\n",
    "\n",
    "for key in unique_nps:\n",
    "    np_to_fact_map[key] = []\n",
    "    \n",
    "question_info = {}\n",
    "\n",
    "total_id = 0\n",
    "\n",
    "for sample in df_dict:\n",
    "    question_info[total_id] = sample[mode[:-1]]\n",
    "    total_id += 1\n",
    "    \n",
    "np_to_fact_map_res = []\n",
    "for np in np_to_fact_map.keys():\n",
    "    np = np.replace(\"(\", \"\").replace(\")\", \"\")\n",
    "    if len(np) < 1:\n",
    "        np = \"---\"\n",
    "    try:\n",
    "        np_to_fact_map_res.append(re.compile(r'(\\b%s\\b)' % np))\n",
    "    except:\n",
    "        np_to_fact_map_res.append(re.compile(r'(\\b%s\\b)' % \"---\"))\n",
    "        \n",
    "        \n",
    "for sample_id in question_info.keys():\n",
    "    for np, np_res in zip(np_to_fact_map.keys(), np_to_fact_map_res):\n",
    "        try:\n",
    "            if re.search(np_res, question_info[sample_id]):\n",
    "                np_to_fact_map[np] += [sample_id]\n",
    "        except:\n",
    "            pass\n",
    "            \n",
    "bad_nps = []\n",
    "\n",
    "for key in np_to_fact_map:\n",
    "    if len(np_to_fact_map[key]) == 0:\n",
    "        bad_nps.append(key)\n",
    "        \n",
    "for np in bad_nps:\n",
    "    np_to_fact_map.pop(np)\n",
    "    \n",
    "print(f\"Number of nounphrases: {len(np_to_fact_map)}\")\n",
    "\n",
    "with open(f\"{mode}_map.json\", \"w\") as f:\n",
    "    json.dump(question_info, f, indent=2)\n",
    "    \n",
    "with open(f\"nounphrases_{mode}_map.json\", \"w\") as f:\n",
    "    json.dump(np_to_fact_map, f, indent=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Py36Agent",
   "language": "python",
   "name": "py36_agent"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
